package com.second.sixoo;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.Decompressor;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.util.ReflectionUtils;
import org.junit.Test;

import java.io.*;


public class HDFSGZIPCompress {
    @Test
    public void Compress() throws Exception {
        testCompress();
//        testUnGzip();
    }

    /**
     * 测试压缩
     *
     * @throws IOException
     * @throws FileNotFoundException
     */
    public void testCompress() throws Exception {
        Class<GzipCodec> clazz = GzipCodec.class;
//        Class<SnappyCodec> clazz = SnappyCodec.class;
//        Class<Lz4Codec> clazz = Lz4Codec.class;
//        Class<BZip2Codec> clazz = BZip2Codec.class;
        Configuration conf = new Configuration();
        CompressionCodec codec = ReflectionUtils.newInstance(clazz, conf);
        //扩展名
        String ext = codec.getDefaultExtension();
        long start = System.currentTimeMillis();
        //得到压缩输出流
        OutputStream out = codec.createOutputStream(new FileOutputStream("/home/hdfs/soft/dec-project/20181119/11" + ext));
        IOUtils.copyBytes(new FileInputStream("/home/hdfs/soft/dec-project/20181119/1.dat"), out, 1024);
        out.close();
        System.out.println(ext + " decompress time : " + (System.currentTimeMillis() - start));
    }


    /**
     * 测试解压缩
     *
     * @throws FileNotFoundException
     * @throws IOException
     */
    @Test
    public void testUnGzip() throws FileNotFoundException, IOException {
        Configuration conf = new Configuration();
        Class<GzipCodec> clazz = GzipCodec.class;
        //创建gzipo codec实例
        CompressionCodec codec = ReflectionUtils.newInstance(clazz, conf);
        //扩展名
        String ext = codec.getDefaultExtension();
        long start = System.currentTimeMillis();
        //解压器
        Decompressor dcor = codec.createDecompressor();
        //得到压缩输出流
        InputStream in = codec.createInputStream(new FileInputStream("/home/hdfs/soft/IdeaProjects/dec-kks-etl/fly.gzip" + ext), dcor);
        IOUtils.copyBytes(in, new FileOutputStream("/home/hdfs/soft/IdeaProjects/dec-kks-etl/fly1" + ext + ".text"), 1024);
        in.close();
        System.out.println(System.currentTimeMillis() - start);
    }

}
